#!usr/bin/env python
#-*-coding:utf-8 -*-
"""
数据源：巨潮 -> 市场资讯 -> 上市公司 -> 公司资讯 -> 股本结构
爬取类目：股本结构
股票类型不包括港股（港股没有最新资料）
url_sample: http://www.cninfo.com.cn/information/stockstructure/szmb000001.html
"""

import sys
import json
from bs4 import BeautifulSoup
from xtls.logger import get_logger

from company_crawler import CompanyCrawler


reload(sys)
sys.setdefaultencoding('utf-8')

def get_single_result():
    RESULT_SINGLE = {
        'changeDate': '',
        'changeReason': '',
        'outstandingShares': {
            'value': '',
            'detail': {
                'commonShares': '',
                'bShares': '',
                'foreignShares': '',
                'otherOutstanding': '',
            },
        },
        'limitShares': {
            'value': '',
            'detail': {
                'limStateShares': '',
                'otherDomesticShares': '',
                'foreignHoldShares': '',
                'allotLegalShares': '',
            },
        },
        'nonOutstandingShares': {
            'value': '',
            'detail': {
                'nonStateShares': '',
                'domensticLegalShares': '',
                'foreignLegalShares': '',
                'naturalShares': '',
                'otherNonOutstandingShares': ''
            },
        },
        'totalShares': '',
    }
    return RESULT_SINGLE

def get_detail_mapping():
    DETAIL_MAPPING = {
        u'变动日期': 'changeDate',
        u'变动原因': 'changeReason',
        u'已流通股份': 'outstandingShares',
        u'人民币普通股': 'commonShares',
        u'境内上市外资股（B股）': 'bShares',
        u'境外上市外资股': 'foreignShares',
        u'其他流通股': 'otherOutstanding',
        u'流通受限股份': 'limitShares',
        u'国有股': 'stateShares',
        u'国有股以外的内资股': 'otherDomesticShares',
        u'外资持股': 'foreignHoldShares',
        u'配售法人股': 'allotLegalShares',
        u'未流通股份': 'nonOutstandingShares',
        u'境内法人持股': 'domensticLegalShares',
        u'境外法人持股': 'foreignLegalShares',
        u'自然人持股': 'naturalShares',
        u'其他未流通股': 'otherNonOutstandingShares',
        u'总股本': 'totalShares',
    }
    return DETAIL_MAPPING

def get_result_format():
    RESULT_FORMAT = {
        'stockStruct': [get_single_result() for _ in xrange(3)]
    }
    return RESULT_FORMAT

def get_first_title():
    FIRST_TITLE = {u'变动日期', u'变动原因', u'总股本', u'已流通股份', u'流通受限股份', u'未流通股份'}
    return FIRST_TITLE

def get_has_subtitle_title():
    HAS_SUBTITLE = {u'已流通股份', u'流通受限股份', u'未流通股份'}
    return HAS_SUBTITLE

class CompanyStockStruc(CompanyCrawler):
    def __init__(self, stock_list):
        logger = get_logger(__file__)
        super(CompanyStockStruc, self).__init__(stock_list=stock_list, logger=logger)
        self.category = 'stockstructure'
        self.tab_name = 'stockstructure'
        self.detail_mapping = get_detail_mapping()

    def find_company_info_from(self, url):

        result = get_result_format()
        FIRST_TITLE = get_first_title()
        HAS_SUBTITLE = get_has_subtitle_title()

        html = self.get(url)

        if not html:
            return result

        soup = BeautifulSoup(html, 'html5lib')

        if soup.find('img', src='/error.jpg'):
            self.logger.info(u'undealed: not find the company %s' % url)
            return result

        tr_list = soup.select('div.clear tr')

        if not tr_list:
            self.logger.info(u'undealed: the page format has been changed %s' % url)
            return result

        for tr in tr_list:
            tds = tr.find_all('td')

            if len(tds) != 4:
                self.logger.warn(u'find the different page format %s' % url)
                continue

            title = tds[0].getText()
            key = self.detail_mapping.get(title, None)

            if not key:
                self.logger.info(u'undealed: Undefined %s' % title)
                continue

            if title not in FIRST_TITLE:
                for index, td in enumerate(tds[1:]):
                    result['stockStruct'][index][parent_key]['detail'][key] = td.getText()

            elif title in HAS_SUBTITLE:
                parent_key = key
                for index, td in enumerate(tds[1:]):
                    result['stockStruct'][index][key]['value'] = td.getText()
            else:
                parent_key = key

                for index, td in enumerate(tds[1:]):
                    result['stockStruct'][index][key] = td.get_text()
        return result

    def _multi_struct_to_simple_struct(self, single):
        for key in ['outstandingShares', 'limitShares', 'nonOutstandingShares']:
            value_dict = single.pop(key, dict())
            single[key] = value_dict.get('value')
            detail = value_dict.get('detail')
            if 'stateShares' in detail.keys():
                state_shares = detail.pop('stateShares')
                state_key_name = key[:3] + 'StateShares'
                detail[state_key_name] = state_shares
            single.update(detail)
        return single

    def clean_data(self, dirty_data):
        self.cleaner.set_dirty_data(dirty_data)
        keys_wan2int = dirty_data.keys()
        keys_wan2int.remove('changeReason')
        keys_wan2int.remove('changeDate')
        self.cleaner.clean_ten_thousand_to_int(keys_wan2int)
        self.cleaner.clean_get_id_single(['changeReason'], conn=self.conn)
        return self.cleaner.complete()

    def save_update(self, data):
        if not data.get('stockStruct', None):
            self.logger.warn('No Save: Code-%s %s has no information' % (data.get('stock_code'), self.category))
            return
        if not data.get('stock_id'):
            return

        version_num = 1 + self.select_current_version_num(table_name='stockstructure',
                                                          stock_id=data.get('stock_id'))
        for single in data.get('stockStruct', dict()):
            single = self._multi_struct_to_simple_struct(single)

            single_clean = self.clean_data(single)
            single_clean['stockId'] = data.get('stock_id', None)
            single_clean['versionNum'] = version_num
            self.insert_to_db(table_name='stockstructure', cleaned_data=single_clean)

    def save_insert(self, data):
        if not data.get('stockStruct', None):
            self.logger.warn('No Save: Code-%s %s has no information' % (data.get('stock_code'), self.category))
            return

        if not data.get('stock_id'):
            return
        lastest_date = self.select_lastest_date(table_name='stockstructure',
                                                field_name='change_date',
                                                stock_id=data.get('stock_id'))

        version_num = self.select_current_version_num(table_name='stockstructure',
                                                      stock_id=data.get('stock_id'))
        version_num = 1 if version_num == 0 else version_num

        for single in data.get('stockStruct', dict()):
            single = self._multi_struct_to_simple_struct(single)
            single_clean = self.clean_data(single)

            cur_date = single_clean.get('changeDate')

            if not self.check_lastest(cur_date, lastest_date):
                continue
            single_clean['stockId'] = data.get('stock_id', None)
            single_clean['versionNum'] = version_num
            self.insert_to_db(table_name='stockstructure', cleaned_data=single_clean)

    def save_print(self, data):
        print json.dumps(data, indent=4)
        print "=" * 70

def main():
    pass

if __name__ == '__main__':
    main()
